﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using System.Web;

namespace MoGuJieSprider
{
    public static class HtmlBuilder
    {

        public static string GetDataFromUrl(string url)
        {
            HttpStatusCode x;
            return GetDataFromUrl(url, out x);
        }

        /// <summary>
        /// 获取指定网页的源码，支持编码自动识别
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetDataFromUrl(string url, out HttpStatusCode statusCode)
        {
            string str = string.Empty;
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

            //设置http头
            request.AllowAutoRedirect = true;
            request.AllowWriteStreamBuffering = false;
            request.Referer = "";
            request.Timeout = 10 * 3000;
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
            request.ServicePoint.ConnectionLimit = 300;
            HttpWebResponse response = null;
            try
            {
                response = (HttpWebResponse)request.GetResponse();

                statusCode = response.StatusCode;
                if (response.StatusCode == HttpStatusCode.OK)
                {
                    //根据http应答的http头来判断编码
                    string characterSet = response.CharacterSet;
                    Encoding encode;
                    if (characterSet != "" && characterSet != "tis-620,text/html")
                    {
                        if (characterSet == "ISO-8859-1")
                        {
                            characterSet = "gb2312";
                        }
                        encode = Encoding.GetEncoding(characterSet);
                    }
                    else
                    {
                        encode = Encoding.Default;
                    }

                    //声明一个内存流来保存http应答流
                    Stream receiveStream = response.GetResponseStream();
                    MemoryStream mStream = new MemoryStream();

                    byte[] bf = new byte[255];
                    int count = receiveStream.Read(bf, 0, 255);
                    while (count > 0)
                    {
                        mStream.Write(bf, 0, count);
                        count = receiveStream.Read(bf, 0, 255);
                    }
                    receiveStream.Close();

                    mStream.Seek(0, SeekOrigin.Begin);

                    //从内存流里读取字符串
                    StreamReader reader = new StreamReader(mStream, encode);
                    char[] buffer = new char[1024];
                    count = reader.Read(buffer, 0, 1024);
                    while (count > 0)
                    {
                        str += new String(buffer, 0, count);
                        count = reader.Read(buffer, 0, 1024);
                    }

                    //从解析出的字符串里判断charset，如果和http应答的编码不一直
                    //那么以页面声明的为准，再次从内存流里重新读取文本
                    Regex reg =
                        new Regex(@"<meta[\s\S]+?charset=(.*)""[\s\S]+?>",
                                  RegexOptions.Multiline | RegexOptions.IgnoreCase);
                    MatchCollection mc = reg.Matches(str);
                    if (mc.Count > 0)
                    {
                        string tempCharSet = mc[0].Result("$1");
                        if (string.Compare(tempCharSet, characterSet, true) != 0)
                        {
                            encode = Encoding.GetEncoding(tempCharSet);
                            str = string.Empty;
                            mStream.Seek(0, SeekOrigin.Begin);
                            reader = new StreamReader(mStream, encode);
                            buffer = new char[255];
                            count = reader.Read(buffer, 0, 255);
                            while (count > 0)
                            {
                                str += new String(buffer, 0, count);
                                count = reader.Read(buffer, 0, 255);
                            }
                        }
                    }
                    reader.Close();
                    mStream.Close();
                }
            }
            catch (Exception ex)
            {
                statusCode = HttpStatusCode.InternalServerError;
                Console.WriteLine(ex.Message);
            }
            finally
            {
                if (response != null)
                    response.Close();
            }
            statusCode = HttpStatusCode.OK;


            return str;
        }


        public static string GetDataFromUrl(string url, string data, out int statusCode)
        {

            byte[] bs = Encoding.UTF8.GetBytes(data);

            string str = string.Empty;
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

            //设置http头
            request.AllowAutoRedirect = true;
            request.AllowWriteStreamBuffering = false;
            request.ContentType = "application/x-www-form-urlencoded";
            request.Referer = "";
            request.Timeout = 10 * 3000;
            request.UserAgent = "";
            request.ServicePoint.ConnectionLimit = 300;
            request.ContentLength = bs.Length;
            request.Method = "post";



            using (Stream reqStream = request.GetRequestStream())
            {
                reqStream.Write(bs, 0, bs.Length);
            }


            HttpWebResponse response = null;


            try
            {
                response = (HttpWebResponse)request.GetResponse();

                statusCode = int.Parse(response.StatusCode.ToString());

                if (response.StatusCode == HttpStatusCode.OK)
                {
                    //根据http应答的http头来判断编码
                    string characterSet = response.CharacterSet;
                    Encoding encode;
                    if (characterSet != "" && characterSet != "tis-620,text/html")
                    {
                        if (characterSet == "ISO-8859-1")
                        {
                            characterSet = "gb2312";
                        }
                        encode = Encoding.GetEncoding(characterSet);
                    }
                    else
                    {
                        encode = Encoding.Default;
                    }

                    //声明一个内存流来保存http应答流
                    Stream receiveStream = response.GetResponseStream();
                    MemoryStream mStream = new MemoryStream();

                    byte[] bf = new byte[255];
                    int count = receiveStream.Read(bf, 0, 255);
                    while (count > 0)
                    {
                        mStream.Write(bf, 0, count);
                        count = receiveStream.Read(bf, 0, 255);
                    }
                    receiveStream.Close();

                    mStream.Seek(0, SeekOrigin.Begin);

                    //从内存流里读取字符串
                    StreamReader reader = new StreamReader(mStream, encode);
                    char[] buffer = new char[1024];
                    count = reader.Read(buffer, 0, 1024);
                    while (count > 0)
                    {
                        str += new String(buffer, 0, count);
                        count = reader.Read(buffer, 0, 1024);
                    }

                    //从解析出的字符串里判断charset，如果和http应答的编码不一直
                    //那么以页面声明的为准，再次从内存流里重新读取文本
                    Regex reg =
                        new Regex(@"<meta[\s\S]+?charset=(.*)""[\s\S]+?>",
                                  RegexOptions.Multiline | RegexOptions.IgnoreCase);
                    MatchCollection mc = reg.Matches(str);
                    if (mc.Count > 0)
                    {
                        string tempCharSet = mc[0].Result("$1");
                        if (string.Compare(tempCharSet, characterSet, true) != 0)
                        {
                            encode = Encoding.GetEncoding(tempCharSet);
                            str = string.Empty;
                            mStream.Seek(0, SeekOrigin.Begin);
                            reader = new StreamReader(mStream, encode);
                            buffer = new char[255];
                            count = reader.Read(buffer, 0, 255);
                            while (count > 0)
                            {
                                str += new String(buffer, 0, count);
                                count = reader.Read(buffer, 0, 255);
                            }
                        }
                    }
                    reader.Close();
                    mStream.Close();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            finally
            {
                if (response != null)
                    response.Close();
            }
            statusCode = 404;
            return str;
        }

        public static List<string> GetTags(this string _Cont, string regex, string tag)
        {

            Regex r = new Regex(regex, RegexOptions.IgnoreCase);

            MatchCollection ms = r.Matches(_Cont);
            List<string> result = new List<string>();
            foreach (Match m in ms)
            {
                result.Add(m.Groups[tag].Value);
            }
            return result;
        }

        public static List<string> GetTags_Div(this string _Cont, string @class)
        {

            string StrRegex = string.Format(@"(?is)<div.*?class=""{0}""[^>]*>(?<C>.*?)</div>", @class);
            Regex r = new Regex(StrRegex, RegexOptions.IgnoreCase);

            MatchCollection ms = r.Matches(_Cont);
            List<string> result = new List<string>();
            foreach (Match m in ms)
            {
                result.Add(m.Groups["C"].Value);
            }


            return result;
        }


        public static string SaveWebPicture(this string url)
        {
            string path = "D:/wwwroot/cgfreight_N/UploadFiles/AutoDownload/";
            if (Directory.Exists(path))
            {
                if (url.Substring(0, 7).ToLower() == "http://" && !url.Contains("www.cgfreight.cn"))
                {
                    string fileext = "." + url.Substring(url.LastIndexOf(".") + 1);

                    string filename = Guid.NewGuid().ToString() + fileext;
                    path = path + filename;
                    WebClient myWebClient = new WebClient();
                    try
                    {
                        myWebClient.DownloadFile(url, path);
                        FileInfo pic = new FileInfo(path);
                        if (pic.Length < 1024)
                        {
                            System.IO.File.Delete(path);

                        }
                        else
                        {
                            url = "http://www.cgfreight.cn/UploadFiles/AutoDownload/" + filename;
                        }
                    }
                    catch { }
                }
            }
            return url;
        }

        public static string GetWebPicture(this string content)
        {
            string path = "D:/wwwroot/cgfreight_N/UploadFiles/AutoDownload/";
            if (Directory.Exists(path))
            {
                Regex rParseImage = new Regex(@"(<img[^>]*((/>)|(>\s*</img>)))", RegexOptions.IgnoreCase | RegexOptions.Singleline);
                Regex rParseImageUrl = new Regex(@"src\s*=\s*(?:([""'])(?<src>[^""']+)\1|(?<src>[^\s>]+))", RegexOptions.IgnoreCase | RegexOptions.Singleline);
                MatchCollection mc = rParseImage.Matches(content);
                foreach (Match m in mc)
                {
                    Match mu = rParseImageUrl.Match(m.Value);
                    string url = mu.Groups["src"].Value;
                    string filepath = url;
                    if (!string.IsNullOrEmpty(url))
                    {
                        if (url.Substring(0, 7).ToLower() == "http://" && !url.Contains("www.cgfreight.cn"))
                        {
                            string fileext = "." + url.Substring(url.LastIndexOf(".") + 1);

                            string filename = Guid.NewGuid().ToString() + fileext;
                            path = path + filename;
                            filepath = "http://www.cgfreight.cn/UploadFiles/AutoDownload/" + filename;
                            WebClient myWebClient = new WebClient();
                            try
                            {
                                myWebClient.DownloadFile(url, path);
                                FileInfo pic = new FileInfo(path);
                                if (pic.Length < 1024)
                                {
                                    System.IO.File.Delete(path);
                                    content = content.Replace(m.Value, "");
                                }
                                else
                                {
                                    content = content.Replace(url, filepath);
                                }
                            }
                            catch { }
                        }
                    }
                }
                return content;
            }
            return content;
        }



        public static HtmlNodeCollection GetHtmlNodes(string html, string xpath)
        {

            var hdoc = new HtmlAgilityPack.HtmlDocument() { OptionWriteEmptyNodes = true };
            hdoc.LoadHtml(html);

            return hdoc.DocumentNode.SelectNodes(xpath);
        }

        public static string RemoveHtml(this string content)
        {
            return Regex.Replace(content, @"<[^>]*>", string.Empty, RegexOptions.IgnoreCase);
        }

        public static string GetTextFromHTML(this string s)
        {
            System.Text.RegularExpressions.Regex regEx = new System.Text.RegularExpressions.Regex(@"</?(?!br|/?p)[^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);

            return regEx.Replace(s, "");
        }
    }
}
